program ChromeIssues;

include <core.vhs>
include <netutil.vhs>

import core::*;
import core::Logging::*;
import core::Pattern::*;
import core::Logging::*;
import core::JobEngine::*;
import netutil::*;	

const CHROME_ISSUE_URL_PATTERN = @'http://code.google.com/p/chromium/issues/detail\?id=(\d+)';
const CHROME_ISSUE_URL = @'http://code.google.com/p/chromium/issues/detail?id=';

var numEntry = 0;
var failDownload = 0;
var localNumEntry;
var localFailDownload;

pattern<DataPart> DownloadIssueEntry($dataPattern=@'^"(\d+)",.+?$', $minOccur=0, $maxOccur = INFINITY, $failIssues) {
	numEntry = numEntry + 1;
	var url = CHROME_ISSUE_URL + this[1];
	log("(%d) Download %s", numEntry, url);
	var ifile = download(url, timeout=500000, expiredTimeSec=0);
	if (ifile == null) {
		log("Unable to download %s", url);
		failDownload = failDownload + 1;
		failIssues.writeCsvLine([this[1], url]);
	}
}

function main() {
	//downloadIssueFile();
	//redownloadIssueFile('output/failissues.csv', 'output/failissues.2.csv');
	//redownloadIssueFile('output/failissues.2.csv', 'output/failissues.3.csv');
	//redownloadIssueFile('output/failissues.3.csv', 'output/failissues.4.csv');
	//redownloadIssueFile('gc.nvd.Nbug.csv', 'output/gc-hidden.csv');
	//redownloadIssueFile('output/gc-hidden.csv', 'output/gc-hidden.2.csv');
	redownloadIssueFile('output/gc-hidden.6.csv', 'output/gc-hidden.7.csv');
}


function redownloadIssueFile(filename='output/failissues.csv', output='output/failissues.2.csv') {
	core::Logging::configureLog("issuecrawler.log");
	
	netutil::registerNameResolver(pattern<DataPart>($dataPattern=CHROME_ISSUE_URL_PATTERN) 'gc-issue.html');
	
	/*var sucess = download('http://code.google.com/p/chromium/issues/detail?id=57061', cacheFile='gcissue.html', timeout=50000);
	if (sucess == null)
		log("unable to download");
	
	return;*/
	
	log("Starting issue crawler");
	var failIssues = new CsvFile(output);
	failIssues.writeln('ID,link');
	numEntry = 0;
	
	log("Process list file: ", filename);
	DownloadIssueEntry(dataPattern=@'^(\d+),.+?$', input = new SequenceStream(filename, 4096), failIssues=failIssues);

	log("Complete");
	log("Summary: %d entries, %d not downloaded", numEntry, failDownload)
}

function downloadIssueFile() {
	core::Logging::configureLog("issuecrawler.log");
	
	netutil::registerNameResolver(pattern<DataPart>($dataPattern=CHROME_ISSUE_URL_PATTERN) 'gc-issue.html');
	
	/*var sucess = download('http://code.google.com/p/chromium/issues/detail?id=57061', cacheFile='gcissue.html', timeout=50000);
	if (sucess == null)
		log("unable to download");
	
	return;*/
	
	log("Starting issue crawler");
	var failIssues = new CsvFile('output/failissues.csv');
	failIssues.writeln('ID,link');
	numEntry = 0;
	for (var i = 1; i <= 115; i = i + 1) {
		localNumEntry = 0;
		localFailDownload = 0;
		log("Process list file: input/%d.csv", i);
		DownloadIssueEntry(input = new SequenceStream('input/' + i + '.csv', 4096), failIssues=failIssues);
		log("Summary of file input/%d.csv: %d entries, %d not downloaded", i, localNumEntry, localFailDownload)
	}
	log("Complete");
	log("Summary: %d entries, %d not downloaded", numEntry, failDownload)
}

function downloadIssueList() {	
	configureLog("test.log");
	var url = "http://code.google.com/p/chromium/issues/csv?can=1&num=1000&q=&start=";
	for (var i = 17001; i < 117754; i = i + 1000) {
		log("Download %s", url + i);
		download(url + i, timeout=500000);
	}
	log("completed");
}


